In [6]:
fig, ax = plt.subplots(figsize=(20, 10))
sn.heatmap(corr_matrix,annot=True)
ax.set_title('Correlation Heatmap Table1')
plt.show()
#Importing Libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sn
import datetime
%matplotlib inline
df=pd.read_csv("Table2014_2023F.csv")
columns_to_convert = ['Solar generation (MW)','Embbeded solar generation (MW)','Embbeded wind generation (MW)','National demand (MW)','Total system demand (MW)','Market Index Price (£/MWh)','Market Index Volume (MWh)','Wind generation (MW)','Hydro generation (MW)','Biomass generation (MW)','System Buy Price(GBP/MWh)']
for column in columns_to_convert:
df[column] = pd.to_numeric(df[column], errors='coerce').astype('float32')
df.describe()
| System Buy Price(GBP/MWh) | Market Index Price (£/MWh) | Market Index Volume (MWh) | Settlement Period | National demand (MW) | Embbeded wind generation (MW) | Embbeded solar generation (MW) | Total system demand (MW) | Wind generation (MW) | Solar generation (MW) | Hydro generation (MW) | Biomass generation (MW) | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| count | 157338.000000 | 157338.000000 | 157338.000000 | 157338.000000 | 157338.000000 | 157338.000000 | 157338.000000 | 157338.000000 | 157338.000000 | 157338.000000 | 157338.000000 | 157338.000000 |
| mean | 70.108925 | 67.942780 | 872.352844 | 24.499123 | 29932.853516 | 1501.433960 | 1175.537231 | 31373.070312 | 5688.662109 | 1157.449951 | 418.494019 | 1110.033325 |
| std | 89.361053 | 70.670906 | 479.337952 | 13.852979 | 7182.784180 | 972.260681 | 1869.287964 | 6932.190430 | 4067.088379 | 1870.588501 | 254.493698 | 1077.196167 |
| min | 0.000000 | 0.000000 | 0.000000 | 1.000000 | 13367.000000 | 83.000000 | 0.000000 | 16629.000000 | 103.000000 | 0.000000 | 0.000000 | 0.000000 |
| 25% | 32.939999 | 35.522500 | 504.962509 | 12.000000 | 24318.000000 | 731.000000 | 0.000000 | 26018.000000 | 2432.000000 | 0.000000 | 208.000000 | 0.000000 |
| 50% | 46.116501 | 44.650002 | 784.099976 | 24.000000 | 29218.000000 | 1282.000000 | 23.000000 | 30561.000000 | 4734.000000 | 3.000000 | 386.000000 | 1127.000000 |
| 75% | 70.119001 | 62.567500 | 1157.199951 | 36.000000 | 34847.750000 | 2070.000000 | 1810.000000 | 35899.000000 | 7925.000000 | 1739.000000 | 594.000000 | 2083.000000 |
| max | 4037.800049 | 1983.660034 | 3743.350098 | 48.000000 | 79138.000000 | 5634.000000 | 13060.000000 | 80820.000000 | 20912.000000 | 9892.000000 | 1403.000000 | 3262.000000 |
corr_matrix=df.corr()
C:\Users\paulo\AppData\Local\Temp\ipykernel_3192\1544550014.py:1: FutureWarning: The default value of numeric_only in DataFrame.corr is deprecated. In a future version, it will default to False. Select only valid columns or specify the value of numeric_only to silence this warning. corr_matrix=df.corr()
fig, ax = plt.subplots(figsize=(20, 10))
sn.heatmap(corr_matrix,annot=True)
ax.set_title('Correlation Heatmap Table1')
plt.show()
from pandas_profiling import ProfileReport
C:\Users\paulo\AppData\Local\Temp\ipykernel_3192\2274191625.py:1: DeprecationWarning: `import pandas_profiling` is going to be deprecated by April 1st. Please use `import ydata_profiling` instead. from pandas_profiling import ProfileReport
#profile = ProfileReport(df)
#profile
profile = ProfileReport(df, explorative=True)
profile.to_widgets()
Summarize dataset: 0%| | 0/5 [00:00<?, ?it/s]
C:\Users\paulo\anaconda3\envs\tf\lib\site-packages\multimethod\__init__.py:315: FutureWarning: In a future version, `df.iloc[:, i] = newvals` will attempt to set the values inplace instead of always setting a new array. To retain the old behavior, use either `df[df.columns[i]] = newvals` or, if columns are non-unique, `df.isetitem(i, newvals)` return func(*args, **kwargs)
Generate report structure: 0%| | 0/1 [00:00<?, ?it/s]
Render widgets: 0%| | 0/1 [00:00<?, ?it/s]
VBox(children=(Tab(children=(Tab(children=(GridBox(children=(VBox(children=(GridspecLayout(children=(HTML(valu…
profile
profile.to_file("pandas_profile.html")
Render HTML: 0%| | 0/1 [00:00<?, ?it/s]
Export report to file: 0%| | 0/1 [00:00<?, ?it/s]
sn.pairplot(df.sample(1000))
plt.savefig("random.jpg", facecolor="k")
plt.show()
sn.jointplot(x=df['Hydro generation (MW)'],y=df['Market Index Price (£/MWh)'], kind='reg',scatter_kws={"s": 1}, color='black')
plt.figure(figsize=(10, 5))
# Display the plot
plt.savefig("random.jpg", facecolor="k")
plt.show()
<Figure size 1000x500 with 0 Axes>
sn.jointplot(x=df['Wind generation (MW)'],y=df['Market Index Price (£/MWh)'], kind='reg',scatter_kws={"s": 1},color='black')
plt.figure(figsize=(10, 5))
# Display the plot
plt.savefig("random.jpg", facecolor="k")
plt.show()
<Figure size 1000x500 with 0 Axes>
sn.jointplot(x=df['Biomass generation (MW)'],y=df['Market Index Price (£/MWh)'], kind='reg',scatter_kws={"s": 1},color='black')
plt.figure(figsize=(10, 5))
# Display the plot
plt.savefig("random.jpg", facecolor="k")
plt.show()
<Figure size 1000x500 with 0 Axes>
sn.jointplot(x=df['Embbeded solar generation (MW)'],y=df['Market Index Price (£/MWh)'], kind='reg',scatter_kws={"s": 1},color='black')
# Display the plot
plt.savefig("random.jpg", facecolor="k")
plt.show()
sn.jointplot(x=df['Embbeded wind generation (MW)'],y=df['Market Index Price (£/MWh)'], kind='reg',scatter_kws={"s": 1},color='black')
plt.figure(figsize=(10, 5))
# Display the plot
plt.savefig("random.jpg", facecolor="k")
plt.show()
<Figure size 1000x500 with 0 Axes>
sn.jointplot(x=df['Total system demand (MW)'],y=df['Market Index Price (£/MWh)'], kind='reg',scatter_kws={"s": 1},color='black')
plt.figure(figsize=(10, 5))
# Display the plot
plt.savefig("random.jpg", facecolor="k")
plt.show()
<Figure size 1000x500 with 0 Axes>
sn.jointplot(x=df['System Buy Price(GBP/MWh)'],y=df['Market Index Price (£/MWh)'], kind='reg',scatter_kws={"s": 1},color='black')
plt.figure(figsize=(10, 5))
# Display the plot
plt.savefig("random.jpg", facecolor="k")
plt.show()
<Figure size 1000x500 with 0 Axes>
sn.jointplot(x=df['Market Index Volume (MWh)'],y=df['Market Index Price (£/MWh)'], kind='reg',color='black',scatter_kws={"s": 1}, height=6, ratio=3)
# Display the plot
plt.savefig("random.jpg", facecolor="k")
plt.show()
sn.jointplot(x=df['Solar generation (MW)'],y=df['Market Index Price (£/MWh)'], kind='reg',scatter_kws={"s": 1},color='black')
# Display the plot
plt.savefig("random.jpg", facecolor="k")
plt.show()
df.info()
<class 'pandas.core.frame.DataFrame'> RangeIndex: 157338 entries, 0 to 157337 Data columns (total 13 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 DATETIME 157338 non-null object 1 System Buy Price(GBP/MWh) 157338 non-null float32 2 Market Index Price (£/MWh) 157338 non-null float32 3 Market Index Volume (MWh) 157338 non-null float32 4 Settlement Period 157338 non-null int64 5 National demand (MW) 157338 non-null float32 6 Embbeded wind generation (MW) 157338 non-null float32 7 Embbeded solar generation (MW) 157338 non-null float32 8 Total system demand (MW) 157338 non-null float32 9 Wind generation (MW) 157338 non-null float32 10 Solar generation (MW) 157338 non-null float32 11 Hydro generation (MW) 157338 non-null float32 12 Biomass generation (MW) 157338 non-null float32 dtypes: float32(11), int64(1), object(1) memory usage: 9.0+ MB
import seaborn as sns
import statsmodels.api as sm
import pandas as pd
# List of variable pairs for regression analysis
variable_pairs = [("Solar generation (MW)", "Market Index Price (£/MWh)"), ("Wind generation (MW)", "Market Index Price (£/MWh)"),
("Hydro generation (MW)", "Market Index Price (£/MWh)"), ("Biomass generation (MW)", "Market Index Price (£/MWh)"),
("Embbeded solar generation (MW)", "Market Index Price (£/MWh)"), ("Embbeded wind generation (MW)", "Market Index Price (£/MWh)"),
("Market Index Volume (MWh)", "Market Index Price (£/MWh)"), ("Total system demand (MW)", "Market Index Price (£/MWh)"), ("System Buy Price(GBP/MWh)", "Market Index Price (£/MWh)")]
# Iterate over variable pairs
for x_var, y_var in variable_pairs:
# Fit regression model
X = df[[x_var]]
y = df[y_var]
X = sm.add_constant(X)
model = sm.OLS(y, X).fit()
# Print regression coefficients
print(f"Regression results for {y_var} vs. {x_var}:")
print(model.summary())
Regression results for Market Index Price (£/MWh) vs. Solar generation (MW):
OLS Regression Results
======================================================================================
Dep. Variable: Market Index Price (£/MWh) R-squared: 0.001
Model: OLS Adj. R-squared: 0.001
Method: Least Squares F-statistic: 120.1
Date: Tue, 08 Aug 2023 Prob (F-statistic): 6.24e-28
Time: 23:55:57 Log-Likelihood: -8.9314e+05
No. Observations: 157338 AIC: 1.786e+06
Df Residuals: 157336 BIC: 1.786e+06
Df Model: 1
Covariance Type: nonrobust
=========================================================================================
coef std err t P>|t| [0.025 0.975]
-----------------------------------------------------------------------------------------
const 66.7350 0.209 318.626 0.000 66.325 67.146
Solar generation (MW) 0.0010 9.52e-05 10.958 0.000 0.001 0.001
==============================================================================
Omnibus: 150133.598 Durbin-Watson: 0.031
Prob(Omnibus): 0.000 Jarque-Bera (JB): 12762728.117
Skew: 4.437 Prob(JB): 0.00
Kurtosis: 46.221 Cond. No. 2.59e+03
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
[2] The condition number is large, 2.59e+03. This might indicate that there are
strong multicollinearity or other numerical problems.
Regression results for Market Index Price (£/MWh) vs. Wind generation (MW):
OLS Regression Results
======================================================================================
Dep. Variable: Market Index Price (£/MWh) R-squared: 0.020
Model: OLS Adj. R-squared: 0.020
Method: Least Squares F-statistic: 3174.
Date: Tue, 08 Aug 2023 Prob (F-statistic): 0.00
Time: 23:55:57 Log-Likelihood: -8.9163e+05
No. Observations: 157338 AIC: 1.783e+06
Df Residuals: 157336 BIC: 1.783e+06
Df Model: 1
Covariance Type: nonrobust
========================================================================================
coef std err t P>|t| [0.025 0.975]
----------------------------------------------------------------------------------------
const 54.0427 0.303 178.184 0.000 53.448 54.637
Wind generation (MW) 0.0024 4.34e-05 56.338 0.000 0.002 0.003
==============================================================================
Omnibus: 152769.374 Durbin-Watson: 0.032
Prob(Omnibus): 0.000 Jarque-Bera (JB): 14208907.558
Skew: 4.537 Prob(JB): 0.00
Kurtosis: 48.662 Cond. No. 1.20e+04
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
[2] The condition number is large, 1.2e+04. This might indicate that there are
strong multicollinearity or other numerical problems.
Regression results for Market Index Price (£/MWh) vs. Hydro generation (MW):
OLS Regression Results
======================================================================================
Dep. Variable: Market Index Price (£/MWh) R-squared: 0.000
Model: OLS Adj. R-squared: 0.000
Method: Least Squares F-statistic: 12.02
Date: Tue, 08 Aug 2023 Prob (F-statistic): 0.000526
Time: 23:55:57 Log-Likelihood: -8.9320e+05
No. Observations: 157338 AIC: 1.786e+06
Df Residuals: 157336 BIC: 1.786e+06
Df Model: 1
Covariance Type: nonrobust
=========================================================================================
coef std err t P>|t| [0.025 0.975]
-----------------------------------------------------------------------------------------
const 66.9271 0.343 195.188 0.000 66.255 67.599
Hydro generation (MW) 0.0024 0.001 3.467 0.001 0.001 0.004
==============================================================================
Omnibus: 150041.717 Durbin-Watson: 0.031
Prob(Omnibus): 0.000 Jarque-Bera (JB): 12684699.076
Skew: 4.435 Prob(JB): 0.00
Kurtosis: 46.084 Cond. No. 943.
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
Regression results for Market Index Price (£/MWh) vs. Biomass generation (MW):
OLS Regression Results
======================================================================================
Dep. Variable: Market Index Price (£/MWh) R-squared: 0.096
Model: OLS Adj. R-squared: 0.096
Method: Least Squares F-statistic: 1.669e+04
Date: Tue, 08 Aug 2023 Prob (F-statistic): 0.00
Time: 23:55:57 Log-Likelihood: -8.8527e+05
No. Observations: 157338 AIC: 1.771e+06
Df Residuals: 157336 BIC: 1.771e+06
Df Model: 1
Covariance Type: nonrobust
===========================================================================================
coef std err t P>|t| [0.025 0.975]
-------------------------------------------------------------------------------------------
const 45.3923 0.243 186.603 0.000 44.915 45.869
Biomass generation (MW) 0.0203 0.000 129.179 0.000 0.020 0.021
==============================================================================
Omnibus: 150925.828 Durbin-Watson: 0.034
Prob(Omnibus): 0.000 Jarque-Bera (JB): 15947158.536
Skew: 4.390 Prob(JB): 0.00
Kurtosis: 51.533 Cond. No. 2.22e+03
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
[2] The condition number is large, 2.22e+03. This might indicate that there are
strong multicollinearity or other numerical problems.
Regression results for Market Index Price (£/MWh) vs. Embbeded solar generation (MW):
OLS Regression Results
======================================================================================
Dep. Variable: Market Index Price (£/MWh) R-squared: 0.000
Model: OLS Adj. R-squared: 0.000
Method: Least Squares F-statistic: 70.05
Date: Tue, 08 Aug 2023 Prob (F-statistic): 5.82e-17
Time: 23:55:57 Log-Likelihood: -8.9317e+05
No. Observations: 157338 AIC: 1.786e+06
Df Residuals: 157336 BIC: 1.786e+06
Df Model: 1
Covariance Type: nonrobust
==================================================================================================
coef std err t P>|t| [0.025 0.975]
--------------------------------------------------------------------------------------------------
const 67.0054 0.210 318.453 0.000 66.593 67.418
Embbeded solar generation (MW) 0.0008 9.53e-05 8.370 0.000 0.001 0.001
==============================================================================
Omnibus: 150072.686 Durbin-Watson: 0.031
Prob(Omnibus): 0.000 Jarque-Bera (JB): 12725143.339
Skew: 4.435 Prob(JB): 0.00
Kurtosis: 46.155 Cond. No. 2.61e+03
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
[2] The condition number is large, 2.61e+03. This might indicate that there are
strong multicollinearity or other numerical problems.
Regression results for Market Index Price (£/MWh) vs. Embbeded wind generation (MW):
OLS Regression Results
======================================================================================
Dep. Variable: Market Index Price (£/MWh) R-squared: 0.000
Model: OLS Adj. R-squared: 0.000
Method: Least Squares F-statistic: 16.14
Date: Tue, 08 Aug 2023 Prob (F-statistic): 5.89e-05
Time: 23:55:57 Log-Likelihood: -8.9320e+05
No. Observations: 157338 AIC: 1.786e+06
Df Residuals: 157336 BIC: 1.786e+06
Df Model: 1
Covariance Type: nonrobust
=================================================================================================
coef std err t P>|t| [0.025 0.975]
-------------------------------------------------------------------------------------------------
const 69.0479 0.328 210.661 0.000 68.405 69.690
Embbeded wind generation (MW) -0.0007 0.000 -4.017 0.000 -0.001 -0.000
==============================================================================
Omnibus: 149935.388 Durbin-Watson: 0.031
Prob(Omnibus): 0.000 Jarque-Bera (JB): 12656636.853
Skew: 4.430 Prob(JB): 0.00
Kurtosis: 46.036 Cond. No. 3.29e+03
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
[2] The condition number is large, 3.29e+03. This might indicate that there are
strong multicollinearity or other numerical problems.
Regression results for Market Index Price (£/MWh) vs. Market Index Volume (MWh):
OLS Regression Results
======================================================================================
Dep. Variable: Market Index Price (£/MWh) R-squared: 0.082
Model: OLS Adj. R-squared: 0.082
Method: Least Squares F-statistic: 1.403e+04
Date: Tue, 08 Aug 2023 Prob (F-statistic): 0.00
Time: 23:55:57 Log-Likelihood: -8.8648e+05
No. Observations: 157338 AIC: 1.773e+06
Df Residuals: 157336 BIC: 1.773e+06
Df Model: 1
Covariance Type: nonrobust
=============================================================================================
coef std err t P>|t| [0.025 0.975]
---------------------------------------------------------------------------------------------
const 31.1372 0.354 87.834 0.000 30.442 31.832
Market Index Volume (MWh) 0.0422 0.000 118.466 0.000 0.041 0.043
==============================================================================
Omnibus: 150786.423 Durbin-Watson: 0.053
Prob(Omnibus): 0.000 Jarque-Bera (JB): 15503128.246
Skew: 4.394 Prob(JB): 0.00
Kurtosis: 50.829 Cond. No. 2.07e+03
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
[2] The condition number is large, 2.07e+03. This might indicate that there are
strong multicollinearity or other numerical problems.
Regression results for Market Index Price (£/MWh) vs. Total system demand (MW):
OLS Regression Results
======================================================================================
Dep. Variable: Market Index Price (£/MWh) R-squared: 0.006
Model: OLS Adj. R-squared: 0.006
Method: Least Squares F-statistic: 946.3
Date: Tue, 08 Aug 2023 Prob (F-statistic): 3.47e-207
Time: 23:55:58 Log-Likelihood: -8.9273e+05
No. Observations: 157338 AIC: 1.785e+06
Df Residuals: 157336 BIC: 1.785e+06
Df Model: 1
Covariance Type: nonrobust
============================================================================================
coef std err t P>|t| [0.025 0.975]
--------------------------------------------------------------------------------------------
const 43.2128 0.823 52.488 0.000 41.599 44.826
Total system demand (MW) 0.0008 2.56e-05 30.762 0.000 0.001 0.001
==============================================================================
Omnibus: 150304.370 Durbin-Watson: 0.031
Prob(Omnibus): 0.000 Jarque-Bera (JB): 12710569.085
Skew: 4.448 Prob(JB): 0.00
Kurtosis: 46.124 Cond. No. 1.49e+05
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
[2] The condition number is large, 1.49e+05. This might indicate that there are
strong multicollinearity or other numerical problems.
Regression results for Market Index Price (£/MWh) vs. System Buy Price(GBP/MWh):
OLS Regression Results
======================================================================================
Dep. Variable: Market Index Price (£/MWh) R-squared: 0.537
Model: OLS Adj. R-squared: 0.537
Method: Least Squares F-statistic: 1.826e+05
Date: Tue, 08 Aug 2023 Prob (F-statistic): 0.00
Time: 23:55:58 Log-Likelihood: -8.3259e+05
No. Observations: 157338 AIC: 1.665e+06
Df Residuals: 157336 BIC: 1.665e+06
Df Model: 1
Covariance Type: nonrobust
=============================================================================================
coef std err t P>|t| [0.025 0.975]
---------------------------------------------------------------------------------------------
const 27.3043 0.154 177.235 0.000 27.002 27.606
System Buy Price(GBP/MWh) 0.5796 0.001 427.352 0.000 0.577 0.582
==============================================================================
Omnibus: 115772.361 Durbin-Watson: 0.447
Prob(Omnibus): 0.000 Jarque-Bera (JB): 746088907.903
Skew: -1.813 Prob(JB): 0.00
Kurtosis: 340.333 Cond. No. 144.
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.